{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N7UbxgytUFhn"
      },
      "source": [
        "# Customized models and datasets for CV\n",
        "\n",
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/HanXudong/fairlib/blob/main/tutorial/colored_MNIST.ipynb)\n",
        "\n",
        "In this tutorial we will:\n",
        "- Show how to add a model for image classification\n",
        "- Show how to add a dataloader with image preprocessing\n",
        "\n",
        "We will be using the colored MNIST dataset, please see the [origianl](https://colab.research.google.com/github/reiinakano/invariant-risk-minimization/blob/master/invariant_risk_minimization_colored_mnist.ipynb#scrollTo=knP-xNzavgAb&uniqifier=1) demo for more details.\n",
        "\n",
        "## Installation\n",
        "\n",
        "Again, the first step will be installing our libarary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u4z-4H6bU9Is",
        "outputId": "5342b8d7-6ae8-4c1e-ccd1-ed743f907e9b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting fairlib\n",
            "  Downloading fairlib-0.0.1-py3-none-any.whl (61 kB)\n",
            "\u001b[K     |████████████████████████████████| 61 kB 3.6 MB/s \n",
            "\u001b[?25hCollecting transformers\n",
            "  Downloading transformers-4.18.0-py3-none-any.whl (4.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.0 MB 14.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairlib) (4.63.0)\n",
            "Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from fairlib) (0.6.2)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from fairlib) (1.0.2)\n",
            "Requirement already satisfied: seaborn in /usr/local/lib/python3.7/dist-packages (from fairlib) (0.11.2)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from fairlib) (1.10.0+cu111)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from fairlib) (3.13)\n",
            "Collecting pickle5\n",
            "  Downloading pickle5-0.0.12-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (256 kB)\n",
            "\u001b[K     |████████████████████████████████| 256 kB 64.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from fairlib) (1.21.5)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from fairlib) (3.2.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from fairlib) (1.3.5)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->fairlib) (3.0.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->fairlib) (1.4.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->fairlib) (0.11.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->fairlib) (2.8.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib->fairlib) (3.10.0.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib->fairlib) (1.15.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->fairlib) (2018.9)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->fairlib) (1.1.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->fairlib) (3.1.0)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->fairlib) (1.4.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers->fairlib) (21.3)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.5.1-py3-none-any.whl (77 kB)\n",
            "\u001b[K     |████████████████████████████████| 77 kB 5.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers->fairlib) (2019.12.20)\n",
            "Collecting PyYAML\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 50.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers->fairlib) (4.11.3)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.5 MB 48.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers->fairlib) (3.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers->fairlib) (2.23.0)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.49-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 47.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers->fairlib) (3.7.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->fairlib) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->fairlib) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->fairlib) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->fairlib) (2.10)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers->fairlib) (7.1.2)\n",
            "Installing collected packages: PyYAML, tokenizers, sacremoses, huggingface-hub, transformers, pickle5, fairlib\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed PyYAML-6.0 fairlib-0.0.1 huggingface-hub-0.5.1 pickle5-0.0.12 sacremoses-0.0.49 tokenizers-0.11.6 transformers-4.18.0\n"
          ]
        }
      ],
      "source": [
        "!pip install fairlib"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "Xt6TeF2FVD0i"
      },
      "outputs": [],
      "source": [
        "import fairlib"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "8RvXBTGJUDTt"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "import numpy as np\n",
        "from PIL import Image\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.autograd import grad\n",
        "from torchvision import transforms\n",
        "from torchvision import datasets\n",
        "import torchvision.datasets.utils as dataset_utils"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "YlHbWs8QUDTv"
      },
      "outputs": [],
      "source": [
        "def color_grayscale_arr(arr, red=True):\n",
        "    \"\"\"Converts grayscale image to either red or green\"\"\"\n",
        "    assert arr.ndim == 2\n",
        "    dtype = arr.dtype\n",
        "    h, w = arr.shape\n",
        "    arr = np.reshape(arr, [h, w, 1])\n",
        "    if red:\n",
        "        arr = np.concatenate([\n",
        "            arr,\n",
        "            np.zeros((h, w, 2), dtype=dtype)], axis=2)\n",
        "    else:\n",
        "        arr = np.concatenate([\n",
        "            np.zeros((h, w, 1), dtype=dtype),\n",
        "            arr,\n",
        "            np.zeros((h, w, 1), dtype=dtype)], axis=2)\n",
        "    return arr"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "BoPDWwu3UDTw"
      },
      "outputs": [],
      "source": [
        "root = \"./data\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 422,
          "referenced_widgets": [
            "03591eb69c4c4e4cb206b50ee7a20f04",
            "c359274f675a47f391c95666dc2b895b",
            "2e00fdb819f64bada76a0aaeb227ffee",
            "83dc7f3f068f4e94b4a0ee4ae847adc4",
            "f261238955694ef28f1a126bde95cb3a",
            "d367c12acbbf411abc8062017db50ebf",
            "1e74b9dc176e4699a4e2289fbfdddca8",
            "3c1b9bc1eac34136ac4f469bed1657b9",
            "2df81f5b4cc6454f9936fd0c85420054",
            "013929ca9c3f4d6485f9546f42b5ebf7",
            "976c70ed80de4f75a88e5b4e96b55337",
            "3ca33feeb1754b98919c4496f2da75bd",
            "65dd39f405c24b429e399f6574698eb6",
            "1603eba4355a46e9880368c7d3897ef7",
            "62e7f79f3a694ee090b8aa189d235b6a",
            "816b7a24e6334f3aa5670fbac13019da",
            "4cf56ce8a4eb4042a738868bcc7536dd",
            "d6d44a123d4b4ee69dcbe22623048517",
            "883283fb098c4dd5a04aa7b8a2b09c72",
            "fbbae0dfa4eb461ca0d4271f9f0445fb",
            "215b63d0bb1f43f99449b822a37409a7",
            "072b3db410264da2a45c2aea43800d80",
            "16f0e0a83d6248839141efc74caf7369",
            "830fd908972f4b2cb8a90e9630b557a0",
            "fd2bdd89b7914b11b6e3263342f4283f",
            "62580203dd0b4bb0adb89e97cd0aaabc",
            "b8a4e26a95c74fb5aa4f0ca1309f8e35",
            "36771f09904f4667bef18535967c88fc",
            "961e1bfb58e04db79fb2444b79584db2",
            "5979d3c990ae425ebfb42c39cf9ff0b5",
            "d608093e67d84b61ada485ea0825e678",
            "ea8687c6368f423e860e213215e6209c",
            "18a2bcdf5333492b8a5613ac67b1d8c3",
            "faab9ab370f74de2939729d8c2959fd6",
            "b616a7a1e329469f8d6e5ab3d1e7a154",
            "93571cc249894cb984d8e96b8b6961e0",
            "a29fcb4a206543e4bd203a664ae8c501",
            "efacb89fe81f4653bb2bf550999d6b74",
            "39f5167c8c3f403dbeed7fe731d7b16f",
            "5489240ca06247b98e3b476d2aa994ee",
            "4ac07f673f2b470c89eabd90a979402b",
            "1fccbc6dc6634631b3ca89a553e76ef4",
            "aef4e5657f454247b7edc5e930827246",
            "876051a51652431b8dbe3ded0451a326"
          ]
        },
        "id": "5xRQv3M4UDTw",
        "outputId": "b9832b0f-2bdd-4b6c-87a2-a7dc5fc397ee"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "03591eb69c4c4e4cb206b50ee7a20f04",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/9912422 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Extracting ./data/MNIST/raw/train-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ./data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3ca33feeb1754b98919c4496f2da75bd",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/28881 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Extracting ./data/MNIST/raw/train-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "16f0e0a83d6248839141efc74caf7369",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/1648877 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Extracting ./data/MNIST/raw/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "faab9ab370f74de2939729d8c2959fd6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/4542 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Extracting ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "\n"
          ]
        }
      ],
      "source": [
        "train_mnist = datasets.mnist.MNIST(root=root, train=True, download=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a7CJNTP9UDTx",
        "outputId": "0fadbce3-7092-4b68-c575-a6a5d66d7d2f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "60000"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(train_mnist)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 166
        },
        "id": "sSU49yJVUDTy",
        "outputId": "3feed1b0-3286-4d22-8b56-6013ef28903f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f1b9516cb10>"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAACECAYAAACJbXCEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANr0lEQVR4nO3de4xU1R0H8O/PZYEiIKzoFhEBcQVBVOpWUInaVJQaG2usVTSGKA1pBZWWGpE2qTZqsFUTRGrEiGjj+9GwbdpSu0VqI27FZ3l0AXlEcFnELWDxAez++seOd8457MzO3LlzZ86d7ych+zvz25l73N/427tn7kNUFURE5J8jSj0BIiIKhw2ciMhTbOBERJ5iAyci8hQbOBGRp9jAiYg8VVADF5EpItIsIptEZG5Uk6LSYl2Ti7VNFgl7HLiIVAHYAGAygO0A3gQwVVXXRTc9ihvrmlysbfL0KOC5ZwHYpKqbAUBEngVwGYCMb4ae0kt748gCNklR+AL7cUC/lAzpvOsqIjwbrEyoaqa6AnnWlnUtK7tV9Rj3wUIa+BAAHxrj7QAmZHtCbxyJCfLtAjZJUWjSxmzpvOtK3mBt/bWtqwcLaeA5EZEZAGYAQG/0KfbmKCZmXSk5WFe/FPIh5g4AQ43x8anHLKq6WFXrVbW+Gr0K2BzFJO+6xjYzKlS3tWVd/VJIA38TQJ2IjBCRngCuBtAQzbSohFjX5GJtEyb0EoqqHhKRWQCWA6gCsERV10Y2MyoJ1jW5WNvkCX0YYRj9pUb5IWbpNWkj9mlbtqMV8sKjFcpHN0eh5IV1LStvdbWsxTMxiYg8xQZOROQpNnAiIk+xgRMReYoNnIjIU2zgRESeYgMnIvIUGzgRkafYwImIPMUGTkTkKTZwIiJPFf164D6SHvaPpeqYQTk/t/lnw4O4vU+HlRs2clcQ97nRvmTFzgd6BvHb9c9Zud3t+4N4wgtzrNxJP30j57lVuipnfFQez51lxO5V7UcZ8Uwnd58RT3VyXxjxfCd3Z+5TowouLPfAiYg8xQZOROSpRC+hVJ1SZ421V3UQf3T+ACv3+cT0MkXNUfut3Gun20saYf35s35BfO9DU6xc07ing3jLwc+t3PzWyUF83Gu8wucJzrinEZ/j5CYZ8QAnd0VE89luxA86ucuN+FMn954Rr4xoLl5jYfPGPXAiIk+xgRMReYoNnIjIU4m7pVr7Bd8I4gVLF1m5k6t7ut9eVAe13Rqf8+vZQdxjf+afe78dh6xxr93pNXFdvabgefl4S7XxRtzo5PI5aiwKHc74BiPej8w+csb/NeLmgmaU5t0t1VjYXPGWakREScIGTkTkqcQdRtirOf33zFtfDLVyJ1e3Fvz6c1omWuPN/7PP0lw68sUg3tth/wVa++DrobbJAweBbUb8iZOL4i/tJme8xxl/y4gPOLnfRbD9isXCFoR74EREnmIDJyLyFBs4EZGnErcGfqhlZxAvvPdKK3f3lPSxQFXv97Vy7924MONr3rX7tCDedKF9ybL2PS3W+JqzbwzirTfbrzPCOseW8tFmxLc6uUuN+B0n554BbXrXiCc7OfeosbFGfEuW16Q8sbAF6XYPXESWiMguEVljPFYjIq+IyMbU14HFnSZFjXVNLta2cuSyhLIUwBTnsbkAGlW1Dp2H38+NeF5UfEvBuibVUrC2FSGnMzFFZDiAP6rqqalxM4ALVLVFRAYDeFVVR2V5CQDxnImZTdWgo4O4/ZM2K7fl6fQyydrzlli5s+65KYiPXRTuUMBy8tWZmFHVNZYz9rLob8TuheEeMeLpTu46I34ayfDVmZhR1LbUdWVhLZGeiVmrql8t/u4EUBt6WlROWNfkYm0TqOAPMVVVs/2mFpEZAGYAQO/D7llE5SqfupJfstWWdfVL2D3w1tSfYUh93ZXpG1V1sarWq2p9NXqF3BzFJFRdY5sdFSKn2rKufgm7B94AYBo6b9k5DcCyyGZURO273XN10w7uy3ylwrHXrgvijx927qDa0Y4E8bKu+7Lk9mbJ/dCIn3Vy7oXpEsC/2rKw3crlMMJnAKwCMEpEtovIdHS+CSaLyEYAF+Lwey9TmWNdk4u1rRzd7oGr6tQMqdIdTkIFY12Ti7WtHIk7EzOsU27bEMTXj7Pf548PS19p/vwrZ1q5fs+9UdyJUUHuMOIzndz5Rnyhk/trUWZDkbnDiCu4sLwWChGRp9jAiYg8xQZOROSpxN3UOApHnH6KNX6o4dEgXnvgWCt3+/uXW2N9J30bkaF3r7JfOMafdTY+3tQ4CiOd8dtG7N6oZYUzXm3Ei5xcOf3He3dT4yhUQmF5U2MiomRhAyci8hSXUHLQdsPZQfzUL++zciN69M74vLFPzrLGdY+mb/5waPPWaCYXQqUuobjMxa/HnVy/LM+b54yfNOIWlFZFLqG4klhYLqEQESULGzgRkafYwImIPMU18DzpuWdY4/7zt1vjZ05cnvG5o1ekL5M26k77cmrtGzdHMLvccA38cOOc8f3OONu71rw5zN1ObkfoGYXDNXBHUgrLNXAiomRhAyci8hQbOBGRp7gGXqCqWvvU+o+uOimIm25bYOWOMH5fXrvlIiu3d1LmuwVFjWvg3RvgjL9rxO6hxeYP8u9ObnJkM8oN18C74WthuQZORJQsbOBERJ7iEkoRPb/dvhphH0nfOPkzPWDlLr1pdvr7ft9U1HlxCaUwXzpj87ZWh5zcxUb8alFmY+MSSgHKubBcQiEiShY2cCIiT7GBExF5inelz1PHJPtU+g+utC8ne+oZW4PYXPN2LWwbb437LFud4TspDqc54+87428acbb/adY543+EnhFFIuGF5R44EZGn2MCJiDzFJZQuSP2p1njDzemlkEfPfcLKndfbPhwwmy/1YBC/0TbCTnaU/pYfSTfKGd9kxJc7ua/n8brtRuxWsSOP16GQKriw3AMnIvJUtw1cRIaKyAoRWScia0XkltTjNSLyiohsTH0dWPzpUlRY12RiXStLLnvghwDMUdUxACYCmCkiYwDMBdCoqnUAGlNj8gfrmkysawXpdg1cVVuQWgFS1U9FZD2AIQAuA3BB6tueQOcJpbcVZZZF0GPEMGv8wfXHBfEdVz1r5a7ouzvUNua12me+rlwwMYgHPrHK/fZYJbWu7hLnNUY808kND7kN94BP82YtDSFfMypJrWvFFzaDvD7EFJHhAMYDaAJQm3qzAMBOALUZnjMDwAwA6I0+YedJRVRoXak8sa7Jl/OHmCLSF8BLAGar6j4zp51XxOrywjequlhV61W1vhq9CposRS+KusYwTcoT61oZctoDF5FqdL4ZnlLVl1MPt4rIYFVtEZHBAHYVa5Jh9Rh+gjXee+bgIL7qV3+xcj8a8DLCmNMy0Rqv+m36fV+z9F9WbmBHaZdNXL7W1d11HGvEC53c6JDbcK8H+RsjXubkyuSIsoCvdWVh85fLUSgC4DEA61X1ASPVAGBaKp6Gw//zqYyxrsnEulaWXPbAzwVwHYB/i8i7qcfmAZgP4HkRmQ5gG4AfFGeKVCSsazKxrhUkl6NQ/gn77nCmyrk7Q8KwrsnEulYW70+l7zHYPr6obcmRQfzjESut3NR+raG2MWvHpCB++2H7aoSDXlxjjWs+La91bl/VOONHjPgMJ3diyG28bsT3O7nlzvjzkNsgBwsbKZ5KT0TkKTZwIiJPebGEcuBi+5DUAz9pC+J5J/3Jyl30tf2httHanv5b6ryGOVZu9C/+E8Q1e+wlEg+ONCpbE5zxrUZ8lpMbEnIb5l/IC5zcPUYc7l1DXWJhY8M9cCIiT7GBExF5ig2ciMhTXqyBb/2e/Xtmw7gXcnreoj0jrfGClRcFsbTbh8qOvmtLENe12ufbtoOKwb1ZijvOZL0z/oMRu7W6z4j35Pj6VCAWNjbcAyci8hQbOBGRp6TzypLx6C81OkF4Nm+pNWkj9mlbptOt8yYi8b2JKCtVZV2T6a2uLvHLPXAiIk+xgRMReYoNnIjIU2zgRESeYgMnIvIUGzgRkafYwImIPMUGTkTkKTZwIiJPsYETEXkq1lPpReRjANsADAKwO7YNZ1eJcxmmqsdE9WKsa7dY1+hU6ly6rG2sDTzYqMjqrs7rLwXOJTrlNH/OJTrlNH/OxcYlFCIiT7GBExF5qlQNfHGJttsVziU65TR/ziU65TR/zsVQkjVwIiIqHJdQiIg8FWsDF5EpItIsIptEZG6c205tf4mI7BKRNcZjNSLyiohsTH0dGMM8horIChFZJyJrReSWUs0lCqyrNZfE1JZ1teZSlnWNrYGLSBWARQC+A2AMgKkiMiau7acsBTDFeWwugEZVrQPQmBoX2yEAc1R1DICJAGamfhalmEtBWNfDJKK2rOthyrOuqhrLPwBnA1hujG8HcHtc2ze2OxzAGmPcDGBwKh4MoLkEc1oGYHI5zIV1ZW1ZV3/qGucSyhAAHxrj7anHSq1WVVtS8U4AtXFuXESGAxgPoKnUcwmJdc3A89qyrhmUU135IaZBO3+NxnZYjoj0BfASgNmquq+Uc0myUvwsWdviY13jbeA7AAw1xsenHiu1VhEZDACpr7vi2KiIVKPzjfCUqr5cyrkUiHV1JKS2rKujHOsaZwN/E0CdiIwQkZ4ArgbQEOP2M2kAMC0VT0Pn2lZRiYgAeAzAelV9oJRziQDrakhQbVlXQ9nWNeaF/0sAbADwAYCfl+CDh2cAtAA4iM41vekAjkbnp8cbAfwNQE0M85iEzj+13gfwburfJaWYC+vK2rKu/taVZ2ISEXmKH2ISEXmKDZyIyFNs4EREnmIDJyLyFBs4EZGn2MCJiDzFBk5E5Ck2cCIiT/0fhwMeOv6lQOcAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 432x288 with 3 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "fig, ax = plt.subplots(1,3)\n",
        "ax[0].imshow(train_mnist[0][0]) # Original\n",
        "ax[1].imshow(color_grayscale_arr(np.array(train_mnist[0][0]))) # Red\n",
        "ax[2].imshow(color_grayscale_arr(np.array(train_mnist[0][0]), False)) # Green"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "pO7gV0edUDTy"
      },
      "outputs": [],
      "source": [
        "test_mnist = datasets.mnist.MNIST(root=root, train=False, download=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 166
        },
        "id": "IaIUq00PUDT0",
        "outputId": "9a46af05-6b83-4f3e-decc-5880ed5d5952"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f1b950ad650>"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAACECAYAAACJbXCEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAMMElEQVR4nO3df4wU9RnH8c8jHIcH2nKIlAJytqLp2WpMUTFoQqOkqDE0USskJfxBpWlpg61pRBvbpjbWmqrRav8gqcUYom3FRGKt1l79EaM90UaUH/LD3+jBKdgiVUDK0z9u3ZsZb5e93dnZ/c6+Xwm55zvfuZsn91weZr87O2PuLgBAeI5odAIAgOrQwAEgUDRwAAgUDRwAAkUDB4BA0cABIFA1NXAzm2tmm81sm5ktTyspNBZ1zS9qmy9W7XXgZjZC0hZJcyRtl7RW0gJ335heesgadc0vaps/I2v43jMkbXP3VyXJzO6VNE9SyT+GUdbuozWmhkMiDfv0Xx3w/VZieth1NTM+DdYk3L1UXaVh1pa6NpX33H1CcmMtDXyypLci4+2Sziz3DaM1RmfauTUcEmno9Z5y08OuK4JBbcP1xlAba2ngFTGzJZKWSNJoddT7cMhItK7ID+oallrexHxb0tTIeEphW4y7r3D3Ge4+o03tNRwOGRl2XTPLDLU6bG2pa1hqaeBrJU03s+PNbJSk+ZLWpJMWGoi65he1zZmql1Dc/aCZfV/SI5JGSLrT3TeklhkagrrmF7XNn6ovI6zG0dbpvInZeL3eoz2+u9zVCsPC1QrN4zBXoQwLdW0qzw+1rMUnMQEgUDRwAAgUDRwAAkUDB4BA0cABIFA0cAAIFA0cAAJFAweAQNHAASBQNHAACBQNHAACRQMHgEDRwAEgUDRwAAgUDRwAAkUDB4BA0cABIFA0cAAIVNXPxGwWuy4/KzY+buG2Yvxy/8TY3IH9bcV48j1tsbmO7XuL8aEXNqaZIqpwSWJ8eSR+JzG3LxKvSsztiMTbhIajsKniDBwAAkUDB4BABf9U+sVbXouNLx7zflU/5/WDHxbjW9/9Wk05VePZ/mnFeMxNn4nNjex5PtVjhfBU+lcT464qf84HkXhDlT+jFtsj8Y2JuefqcLymfyo9ha0WT6UHgDyhgQNAoGjgABCo4C8jvO2a+bHxT08Z/D9p3Kb4Et77XxpcHhx1yr9jczd++f5ifMuk3tjcXz4cW4wv7NirSn3kB2Lj3v1jivHs0R/Hd44c84TLvhObOrGn4kPmxuWJ8amROHmRZ3ckPi0xNzsSz0zMvRWJp1acmXQwMX43Ek8q831vJsb1WSptchQ2VYc9AzezO82s38zWR7Z1mtmjZra18HVcfdNE2qhrflHb1lHJEspKSXMT25ZL6nH36ZJ6CmOEZaWoa16tFLVtCYddQnH3J82sK7F5ngZfxNwl6XFJV6WYV8XG3NebGJfe9+gyP+e3n5tdjH85qyv+fU8MftTrxtknVJzbyI8OxXN7sa8Yj39ydWzuK6MGPxna8Xr8U6L10Ox1Ta4alVtFerjMXPQ0M/kqPPpK9/RKkirYlxhvicSbEnOdkTh5BV29NHVtKWyqqn0Tc6K7f9KNdkiaWG5nBIO65he1zaGa38R0dy93wb+ZLZG0RJJGq6PWwyEjw6krwlKuttQ1LNWege80s0mSVPjaX2pHd1/h7jPcfUab2qs8HDJSVV0zyw61qKi21DUs1Z6Br5G0SNINha8PpJZRgxzcsbMYj1m9Mzb3v0g85r5dVR9j57cH75x48qj4r/43u08qxl1/iC+qJa9uqqPc1TV6Y4V/lNmvlis1L47EyUs7XorE99ZwjBTkq7YUVlJllxHeI+kZSSeZ2XYzW6yBP4I5ZrZV0nmFMQJCXfOL2raOSq5CWVBiKt27UiFT1DW/qG3rCP6TmM1s5LT4x8Buv+b2YtxmI2Jzf771vGI8vu+Z+iaGmhybGP8uEidf0v4iEu+uTzpIS4CF5V4oABAoGjgABIoGDgCBYg28jl7+4eTY+PT2wbshbjjwUWyuc+OHQhiWJsYTInHyeVCb65wLUhRgYTkDB4BA0cABIFAsoaRs/4WDtz/71yW3JGYHbyXw3WXLYjNHPv1sPdNCjWZF4nL3Yf1GYrx+yL3QNAIvLGfgABAoGjgABIoGDgCBYg08ZW+eP/h/4liL3z53wWtzinHHw+ticyVvvI2mcEEkTj4vKXrDO26CEJjAC8sZOAAEigYOAIGigQNAoFgDr9ERRx0VGy8856livOdQ/DHX/dd/oRi3719b38RQkyMT47mR+EBi7meR+OP6pIO05KywnIEDQKBo4AAQKJZQarT15yfHxg8eM/gYj3lbL47NtT/EskkofpwYnxaJH07MPV3nXJCinBWWM3AACBQNHAACRQMHgECxBj5M//nWzNj4xctui41fOTh4vdHeX0+JzbWrr36JoSYXJsbXJsZ7IvF1dc4FKcp5YTkDB4BA0cABIFAsoVRg5OTPF+Mrrv1jbK7d4r/C+esWFuMJf+WywWY2PhLflpgbkRg/FImb9MZ0+EQLFZYzcAAI1GEbuJlNNbPHzGyjmW0ws2WF7Z1m9qiZbS18HVf/dJEW6ppP1LW1VHIGflDSle7eLWmmpKVm1q2BR4D2uPt0Ddz6vNwjQdF8qGs+UdcWctg1cHfvkwauf3P3D8xsk6TJkuZJml3Y7S5Jj0u6qi5ZZsxGxn8tpz64vRhfOnZXbG7VB8fGxhOvHfw/8VAdcktLK9Y1ufwZ/eT08Ym5VxLj5NVnzaoV69oShS1hWG9imlmXBu4e0CtpYuGPRZJ2SJpY4nuWSFoiSaPVUW2eqKNa64rmRF3zr+I3Mc1srKTVkq5w9+jl73J3V4nHOrr7Cnef4e4z2tQ+1C5ooDTqmkGaGCbq2hoqOgM3szYN/DGscvf7C5t3mtkkd+8zs0mS+uuVZOZOPSk2vO7Yu0vuesf1l8bGn10XzrVIrVbXLybGXy2z748S4+Qr72bWanVtmcIOoZKrUEzS7yVtcvebI1NrJC0qxIskPZB+eqgX6ppP1LW1VHIGPkvSQkkvmdkLhW3XSLpB0p/MbLGkNyR9sz4pok6oaz5R1xZSyVUoT0myEtPnppsOskJd84m6thY+Sl8wovvEYrzk3tKvLrvvXBobd939z7rlhNpNi8R/K7Nf8kEtD9YhF6SIwkrio/QAECwaOAAEiiWUgpe/N3hriIs69pTcb8rjB+IbfMjLadEkop9IOa7Mfk8kxlS1yVFYSZyBA0CwaOAAECgaOAAEqmXXwPdddEZs3HPRTZERN90K1TmJ8Q8akgVSR2GHxBk4AASKBg4AgWrZJZR3ZsXvAn/cyNLLJtGHNrTtiV9GmLOrkoJ3dmI8tsy+0RvR7a1DLkgRhR0SZ+AAECgaOAAEigYOAIFq2TXwcn61qzs2fubrXcXY+17KOBukZV1iHL236u4sE0G6WriwnIEDQKBo4AAQKPMM76Z3tHX6mcZDQRqt13u0x3eXemrLsJkZV1M2CXenrvn0vLvPSG7kDBwAAkUDB4BA0cABIFCZroGb2buS3pB0jKT3Mjtwea2YyzR3n5DWD6Ouh0Vd09OquQxZ20wbePGgZs8NtSDfCOSSnmbKn1zS00z5k0scSygAECgaOAAEqlENfEWDjjsUcklPM+VPLulppvzJJaIha+AAgNqxhAIAgcq0gZvZXDPbbGbbzGx5lscuHP9OM+s3s/WRbZ1m9qiZbS18HZdBHlPN7DEz22hmG8xsWaNySQN1jeWSm9pS11guTVnXzBq4mY2QdIek8yV1S1pgZt3lvyt1KyXNTWxbLqnH3adL6imM6+2gpCvdvVvSTElLC7+LRuRSE+r6KbmoLXX9lOasq7tn8k/SWZIeiYyvlnR1VsePHLdL0vrIeLOkSYV4kqTNDcjpAUlzmiEX6kptqWs4dc1yCWWypLci4+2FbY020d37CvEOSROzPLiZdUk6TVJvo3OpEnUtIfDaUtcSmqmuvIkZ4QP/jWZ2WY6ZjZW0WtIV7r6nkbnkWSN+l9S2/qhrtg38bUlTI+MphW2NttPMJklS4Wt/Fgc1szYN/CGscvf7G5lLjahrQk5qS10TmrGuWTbwtZKmm9nxZjZK0nxJazI8filrJC0qxIs0sLZVV2Zmkn4vaZO739zIXFJAXSNyVFvqGtG0dc144f8CSVskvSLpJw144+EeSX2SPtbAmt5iSeM18O7xVkl/l9SZQR5na+Cl1ouSXij8u6ARuVBXaktdw60rn8QEgEDxJiYABIoGDgCBooEDQKBo4AAQKBo4AASKBg4AgaKBA0CgaOAAEKj/A2eLLmJ5OmohAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 432x288 with 3 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "fig, ax = plt.subplots(1,3)\n",
        "ax[0].imshow(test_mnist[0][0]) # Original\n",
        "ax[1].imshow(color_grayscale_arr(np.array(test_mnist[0][0]))) # Red\n",
        "ax[2].imshow(color_grayscale_arr(np.array(test_mnist[0][0]), False)) # Green"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "DK4bTPeeUDT0"
      },
      "outputs": [],
      "source": [
        "def process_colored_MNIST(loaded_data, ratio=0.2):\n",
        "    im_list, target_class, protected_class = [], [], []\n",
        "    # Iterate over all images\n",
        "    for idx, (im, label) in enumerate(loaded_data):\n",
        "        if idx % 5000 == 0:\n",
        "            print(f'Converting image {idx}/{len(loaded_data)}')\n",
        "        im_array = np.array(im)\n",
        "\n",
        "        # Assign a binary label y to the image based on the digit\n",
        "        if label < 5:\n",
        "            red_prob = ratio\n",
        "        else:\n",
        "            red_prob = 1-ratio\n",
        "        \n",
        "        color_red = (np.random.uniform() < red_prob)\n",
        "\n",
        "        colored_array = color_grayscale_arr(im_array, red=color_red)\n",
        "\n",
        "        im_list.append(Image.fromarray(colored_array))\n",
        "        target_class.append(int(label))\n",
        "        protected_class.append(int(color_red))\n",
        "    \n",
        "    return (im_list, target_class, protected_class)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "VgSlB9N2UDT1"
      },
      "outputs": [],
      "source": [
        "num_train = len(train_mnist)\n",
        "indices = list(range(num_train))\n",
        "np.random.shuffle(indices)\n",
        "train_idx, valid_idx = indices[10000:], indices[:10000]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z2ZTo-0pUDT1",
        "outputId": "e9890662-a59e-4773-eca1-643786305440"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Converting image 0/50000\n",
            "Converting image 5000/50000\n",
            "Converting image 10000/50000\n",
            "Converting image 15000/50000\n",
            "Converting image 20000/50000\n",
            "Converting image 25000/50000\n",
            "Converting image 30000/50000\n",
            "Converting image 35000/50000\n",
            "Converting image 40000/50000\n",
            "Converting image 45000/50000\n",
            "Converting image 0/10000\n",
            "Converting image 5000/10000\n",
            "Converting image 0/10000\n",
            "Converting image 5000/10000\n"
          ]
        }
      ],
      "source": [
        "colored_MNIST_train = process_colored_MNIST(torch.utils.data.Subset(train_mnist, train_idx), ratio = 0.8)\n",
        "colored_MNIST_dev = process_colored_MNIST(torch.utils.data.Subset(train_mnist, valid_idx), ratio = 0.5)\n",
        "colored_MNIST_test = process_colored_MNIST(test_mnist, ratio = 0.5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "P_HXQeWVUDT1"
      },
      "outputs": [],
      "source": [
        "torch.save(colored_MNIST_train, os.path.join(\"./data\", \"colored_MNIST_train.pt\"))\n",
        "torch.save(colored_MNIST_dev, os.path.join(\"./data\", \"colored_MNIST_dev.pt\"))\n",
        "torch.save(colored_MNIST_test, os.path.join(\"./data\", \"colored_MNIST_test.pt\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "3qOYUt8eUDT2"
      },
      "outputs": [],
      "source": [
        "from fairlib import networks, BaseOptions, dataloaders"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "WIQgZgVgUDT2"
      },
      "outputs": [],
      "source": [
        "Shared_options = {\n",
        "    # The name of the dataset, correponding dataloader will be used,\n",
        "    \"dataset\":  \"MNIST\",\n",
        "\n",
        "    # Specifiy the path to the input data\n",
        "    \"data_dir\": \"./data\",\n",
        "\n",
        "    # Device for computing, -1 is the cpu\n",
        "    \"device_id\": -1,\n",
        "\n",
        "    # The default path for saving experimental results\n",
        "    \"results_dir\":  r\"results\",\n",
        "\n",
        "    # The same as the dataset\n",
        "    \"project_dir\":  r\"dev\",\n",
        "\n",
        "    # We will focusing on TPR GAP, implying the Equalized Odds for binay classification.\n",
        "    \"GAP_metric_name\":  \"TPR_GAP\",\n",
        "\n",
        "    # The overall performance will be measured as accuracy\n",
        "    \"Performance_metric_name\":  \"accuracy\",\n",
        "\n",
        "    # Model selections are based on DTO\n",
        "    \"selection_criterion\":  \"DTO\",\n",
        "\n",
        "    # Default dirs for saving checkpoints\n",
        "    \"checkpoint_dir\":   \"models\",\n",
        "    \"checkpoint_name\":  \"checkpoint_epoch\",\n",
        "\n",
        "\n",
        "    \"n_jobs\":   1,\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "k-ts_cMOUDT2"
      },
      "outputs": [],
      "source": [
        "class ConvNet(networks.utils.BaseModel):\n",
        "\n",
        "    def __init__(self, args):\n",
        "        super(ConvNet, self).__init__()\n",
        "        self.args = args\n",
        "        \n",
        "        self.conv1 = nn.Conv2d(3, 20, 5, 1)\n",
        "        self.conv2 = nn.Conv2d(20, 50, 5, 1)\n",
        "\n",
        "        self.classifier = networks.classifier.MLP(args)\n",
        "\n",
        "        self.init_for_training()\n",
        "\n",
        "    def forward(self, input_data, group_label = None):\n",
        "        x = input_data\n",
        "        x = F.relu(self.conv1(x))\n",
        "        x = F.max_pool2d(x, 2, 2)\n",
        "        x = F.relu(self.conv2(x))\n",
        "        x = F.max_pool2d(x, 2, 2)\n",
        "        x = x.view(-1, 4 * 4 * 50)\n",
        "\n",
        "        return self.classifier(x, group_label)\n",
        "    \n",
        "    def hidden(self, input_data, group_label = None):\n",
        "        x = input_data\n",
        "        x = F.relu(self.conv1(x))\n",
        "        x = F.max_pool2d(x, 2, 2)\n",
        "        x = F.relu(self.conv2(x))\n",
        "        x = F.max_pool2d(x, 2, 2)\n",
        "        x = x.view(-1, 4 * 4 * 50)\n",
        "\n",
        "        return self.classifier.hidden(x, group_label)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-fAarStJUDT2",
        "outputId": "86eac2cf-3bd0-454d-ee81-ff43b8c536e4"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:root:Unexpected args: ['-f', '/root/.local/share/jupyter/runtime/kernel-9c3335e9-652e-4f42-97e0-8b86721e4921.json']\n",
            "INFO:root:Logging to ./results/dev/MNIST/vanilla/output.log\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2022-04-08 15:15:04 [INFO ]  ======================================== 2022-04-08 15:15:04 ========================================\n",
            "2022-04-08 15:15:04 [INFO ]  Base directory is ./results/dev/MNIST/vanilla\n",
            "Not implemented\n",
            "2022-04-08 15:15:04 [INFO ]  dataloaders need to be initialized!\n"
          ]
        }
      ],
      "source": [
        "args = {\n",
        "    \"dataset\":Shared_options[\"dataset\"], \n",
        "    \"data_dir\":Shared_options[\"data_dir\"],\n",
        "    \"device_id\":Shared_options[\"device_id\"],\n",
        "\n",
        "    # Give a name to the exp, which will be used in the path\n",
        "    \"exp_id\":\"vanilla\",\n",
        "\n",
        "    \"emb_size\": 4*4*50,\n",
        "    \"num_classes\": 10,\n",
        "    \"num_groups\": 2,\n",
        "}\n",
        "\n",
        "# Init the argument\n",
        "options = BaseOptions()\n",
        "state = options.get_state(args=args, silence=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "1L9WAC7jUDT3"
      },
      "outputs": [],
      "source": [
        "class CustomizedDataset(dataloaders.utils.BaseDataset):\n",
        "\n",
        "    transform=transforms.Compose([\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize((0.1307, 0.1307, 0.), (0.3081, 0.3081, 0.3081))])\n",
        "\n",
        "    def load_data(self):\n",
        "\n",
        "        self.data_dir = os.path.join(self.args.data_dir, \"colored_MNIST_{}.pt\".format(self.split))\n",
        "\n",
        "        data = torch.load(self.data_dir)\n",
        "\n",
        "        self.X = [self.transform(_img) for _img in data[0]]\n",
        "        self.y = data[1]\n",
        "        self.protected_label = data[2]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xXc659mpUDT3",
        "outputId": "093dd735-669a-4fd1-d2f8-c4e31c26a1c9"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/fairlib/src/dataloaders/utils.py:163: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
            "  self.X = np.array(self.X)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded data shapes: (50000,), (50000,), (50000,)\n",
            "Loaded data shapes: (10000,), (10000,), (10000,)\n",
            "Loaded data shapes: (10000,), (10000,), (10000,)\n"
          ]
        }
      ],
      "source": [
        "customized_train_data = CustomizedDataset(args=state, split=\"train\")\n",
        "customized_dev_data = CustomizedDataset(args=state, split=\"dev\")\n",
        "customized_test_data = CustomizedDataset(args=state, split=\"test\")\n",
        "\n",
        "# DataLoader Parameters\n",
        "tran_dataloader_params = {\n",
        "        'batch_size': state.batch_size,\n",
        "        'shuffle': True,\n",
        "        'num_workers': state.num_workers}\n",
        "\n",
        "eval_dataloader_params = {\n",
        "        'batch_size': state.test_batch_size,\n",
        "        'shuffle': False,\n",
        "        'num_workers': state.num_workers}\n",
        "\n",
        "# init dataloader\n",
        "customized_training_generator = torch.utils.data.DataLoader(customized_train_data, **tran_dataloader_params)\n",
        "customized_validation_generator = torch.utils.data.DataLoader(customized_dev_data, **eval_dataloader_params)\n",
        "customized_test_generator = torch.utils.data.DataLoader(customized_test_data, **eval_dataloader_params)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SGfihIUqUDT3",
        "outputId": "d6afe9b8-c118-44af-a730-06aea8ed41cd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2022-04-08 15:15:28 [INFO ]  MLP( \n",
            "2022-04-08 15:15:28 [INFO ]    (output_layer): Linear(in_features=300, out_features=10, bias=True)\n",
            "2022-04-08 15:15:28 [INFO ]    (AF): Tanh()\n",
            "2022-04-08 15:15:28 [INFO ]    (hidden_layers): ModuleList(\n",
            "2022-04-08 15:15:28 [INFO ]      (0): Linear(in_features=800, out_features=300, bias=True)\n",
            "2022-04-08 15:15:28 [INFO ]      (1): Tanh()\n",
            "2022-04-08 15:15:28 [INFO ]      (2): Linear(in_features=300, out_features=300, bias=True)\n",
            "2022-04-08 15:15:28 [INFO ]      (3): Tanh()\n",
            "2022-04-08 15:15:28 [INFO ]    )\n",
            "2022-04-08 15:15:28 [INFO ]    (criterion): CrossEntropyLoss()\n",
            "2022-04-08 15:15:28 [INFO ]  )\n",
            "2022-04-08 15:15:28 [INFO ]  Total number of parameters: 333610 \n",
            "\n",
            "2022-04-08 15:15:28 [INFO ]  ConvNet( \n",
            "2022-04-08 15:15:28 [INFO ]    (conv1): Conv2d(3, 20, kernel_size=(5, 5), stride=(1, 1))\n",
            "2022-04-08 15:15:28 [INFO ]    (conv2): Conv2d(20, 50, kernel_size=(5, 5), stride=(1, 1))\n",
            "2022-04-08 15:15:28 [INFO ]    (classifier): MLP(\n",
            "2022-04-08 15:15:28 [INFO ]      (output_layer): Linear(in_features=300, out_features=10, bias=True)\n",
            "2022-04-08 15:15:28 [INFO ]      (AF): Tanh()\n",
            "2022-04-08 15:15:28 [INFO ]      (hidden_layers): ModuleList(\n",
            "2022-04-08 15:15:28 [INFO ]        (0): Linear(in_features=800, out_features=300, bias=True)\n",
            "2022-04-08 15:15:28 [INFO ]        (1): Tanh()\n",
            "2022-04-08 15:15:28 [INFO ]        (2): Linear(in_features=300, out_features=300, bias=True)\n",
            "2022-04-08 15:15:28 [INFO ]        (3): Tanh()\n",
            "2022-04-08 15:15:28 [INFO ]      )\n",
            "2022-04-08 15:15:28 [INFO ]      (criterion): CrossEntropyLoss()\n",
            "2022-04-08 15:15:28 [INFO ]    )\n",
            "2022-04-08 15:15:28 [INFO ]    (criterion): CrossEntropyLoss()\n",
            "2022-04-08 15:15:28 [INFO ]  )\n",
            "2022-04-08 15:15:28 [INFO ]  Total number of parameters: 360180 \n",
            "\n"
          ]
        }
      ],
      "source": [
        "model = ConvNet(state)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BzHN7paxUDT4",
        "outputId": "abb6c98b-f1a3-482e-d420-aca10b75d1db"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2022-04-08 15:15:31 [INFO ]  Epoch:    0 [      0/  50000 ( 0%)]\tLoss: 2.3092\t Data Time: 0.02s\tTrain Time: 1.06s\n",
            "2022-04-08 15:16:09 [INFO ]  Evaluation at Epoch 0\n",
            "2022-04-08 15:16:09 [INFO ]  Validation accuracy: 95.66\tmacro_fscore: 95.62\tmicro_fscore: 95.66\tTPR_GAP: 4.20\tFPR_GAP: 0.42\tPPR_GAP: 0.89\t\n",
            "2022-04-08 15:16:09 [INFO ]  Test accuracy: 95.89\tmacro_fscore: 95.86\tmicro_fscore: 95.89\tTPR_GAP: 4.93\tFPR_GAP: 0.43\tPPR_GAP: 1.01\t\n",
            "2022-04-08 15:16:10 [INFO ]  Epoch:    1 [      0/  50000 ( 0%)]\tLoss: 0.1109\t Data Time: 0.01s\tTrain Time: 0.64s\n",
            "2022-04-08 15:16:48 [INFO ]  Evaluation at Epoch 1\n",
            "2022-04-08 15:16:48 [INFO ]  Validation accuracy: 97.50\tmacro_fscore: 97.49\tmicro_fscore: 97.50\tTPR_GAP: 2.36\tFPR_GAP: 0.29\tPPR_GAP: 0.70\t\n",
            "2022-04-08 15:16:48 [INFO ]  Test accuracy: 97.70\tmacro_fscore: 97.70\tmicro_fscore: 97.70\tTPR_GAP: 3.01\tFPR_GAP: 0.35\tPPR_GAP: 0.75\t\n",
            "2022-04-08 15:16:48 [INFO ]  Epoch:    2 [      0/  50000 ( 0%)]\tLoss: 0.0479\t Data Time: 0.01s\tTrain Time: 0.63s\n",
            "2022-04-08 15:17:24 [INFO ]  Evaluation at Epoch 2\n",
            "2022-04-08 15:17:24 [INFO ]  Validation accuracy: 98.05\tmacro_fscore: 98.03\tmicro_fscore: 98.05\tTPR_GAP: 1.54\tFPR_GAP: 0.20\tPPR_GAP: 0.58\t\n",
            "2022-04-08 15:17:24 [INFO ]  Test accuracy: 98.44\tmacro_fscore: 98.42\tmicro_fscore: 98.44\tTPR_GAP: 1.42\tFPR_GAP: 0.15\tPPR_GAP: 0.52\t\n",
            "2022-04-08 15:17:25 [INFO ]  Epoch:    3 [      0/  50000 ( 0%)]\tLoss: 0.0367\t Data Time: 0.01s\tTrain Time: 0.64s\n",
            "2022-04-08 15:18:01 [INFO ]  Evaluation at Epoch 3\n",
            "2022-04-08 15:18:01 [INFO ]  Validation accuracy: 98.15\tmacro_fscore: 98.15\tmicro_fscore: 98.15\tTPR_GAP: 2.43\tFPR_GAP: 0.21\tPPR_GAP: 0.73\t\n",
            "2022-04-08 15:18:01 [INFO ]  Test accuracy: 98.32\tmacro_fscore: 98.31\tmicro_fscore: 98.32\tTPR_GAP: 2.52\tFPR_GAP: 0.23\tPPR_GAP: 0.71\t\n",
            "2022-04-08 15:18:02 [INFO ]  Epoch:    4 [      0/  50000 ( 0%)]\tLoss: 0.0225\t Data Time: 0.01s\tTrain Time: 0.64s\n",
            "2022-04-08 15:18:38 [INFO ]  Evaluation at Epoch 4\n",
            "2022-04-08 15:18:38 [INFO ]  Validation accuracy: 98.25\tmacro_fscore: 98.24\tmicro_fscore: 98.25\tTPR_GAP: 1.92\tFPR_GAP: 0.19\tPPR_GAP: 0.57\t\n",
            "2022-04-08 15:18:38 [INFO ]  Test accuracy: 98.48\tmacro_fscore: 98.48\tmicro_fscore: 98.48\tTPR_GAP: 2.00\tFPR_GAP: 0.20\tPPR_GAP: 0.65\t\n",
            "2022-04-08 15:18:38 [INFO ]  Epoch:    5 [      0/  50000 ( 0%)]\tLoss: 0.0089\t Data Time: 0.01s\tTrain Time: 0.64s\n",
            "2022-04-08 15:19:14 [INFO ]  Evaluation at Epoch 5\n",
            "2022-04-08 15:19:14 [INFO ]  Validation accuracy: 98.54\tmacro_fscore: 98.53\tmicro_fscore: 98.54\tTPR_GAP: 1.22\tFPR_GAP: 0.15\tPPR_GAP: 0.58\t\n",
            "2022-04-08 15:19:14 [INFO ]  Test accuracy: 98.98\tmacro_fscore: 98.98\tmicro_fscore: 98.98\tTPR_GAP: 1.70\tFPR_GAP: 0.14\tPPR_GAP: 0.62\t\n",
            "2022-04-08 15:19:15 [INFO ]  Epoch:    6 [      0/  50000 ( 0%)]\tLoss: 0.0104\t Data Time: 0.01s\tTrain Time: 0.64s\n",
            "2022-04-08 15:19:51 [INFO ]  Evaluation at Epoch 6\n",
            "2022-04-08 15:19:51 [INFO ]  Validation accuracy: 98.66\tmacro_fscore: 98.65\tmicro_fscore: 98.66\tTPR_GAP: 1.23\tFPR_GAP: 0.12\tPPR_GAP: 0.54\t\n",
            "2022-04-08 15:19:51 [INFO ]  Test accuracy: 98.83\tmacro_fscore: 98.83\tmicro_fscore: 98.83\tTPR_GAP: 1.27\tFPR_GAP: 0.18\tPPR_GAP: 0.60\t\n",
            "2022-04-08 15:19:52 [INFO ]  Epoch:    7 [      0/  50000 ( 0%)]\tLoss: 0.0110\t Data Time: 0.01s\tTrain Time: 0.64s\n",
            "2022-04-08 15:20:26 [INFO ]  Epochs since last improvement: 1\n",
            "2022-04-08 15:20:29 [INFO ]  Evaluation at Epoch 7\n",
            "2022-04-08 15:20:29 [INFO ]  Validation accuracy: 98.40\tmacro_fscore: 98.39\tmicro_fscore: 98.40\tTPR_GAP: 1.66\tFPR_GAP: 0.16\tPPR_GAP: 0.63\t\n",
            "2022-04-08 15:20:29 [INFO ]  Test accuracy: 98.91\tmacro_fscore: 98.90\tmicro_fscore: 98.91\tTPR_GAP: 1.55\tFPR_GAP: 0.13\tPPR_GAP: 0.57\t\n",
            "2022-04-08 15:20:30 [INFO ]  Epoch:    8 [      0/  50000 ( 0%)]\tLoss: 0.0040\t Data Time: 0.01s\tTrain Time: 0.70s\n",
            "2022-04-08 15:21:03 [INFO ]  Epochs since last improvement: 2\n",
            "2022-04-08 15:21:06 [INFO ]  Evaluation at Epoch 8\n",
            "2022-04-08 15:21:06 [INFO ]  Validation accuracy: 98.44\tmacro_fscore: 98.43\tmicro_fscore: 98.44\tTPR_GAP: 1.91\tFPR_GAP: 0.18\tPPR_GAP: 0.58\t\n",
            "2022-04-08 15:21:06 [INFO ]  Test accuracy: 98.80\tmacro_fscore: 98.79\tmicro_fscore: 98.80\tTPR_GAP: 2.21\tFPR_GAP: 0.22\tPPR_GAP: 0.60\t\n",
            "2022-04-08 15:21:06 [INFO ]  Epoch:    9 [      0/  50000 ( 0%)]\tLoss: 0.0037\t Data Time: 0.01s\tTrain Time: 0.63s\n",
            "2022-04-08 15:21:40 [INFO ]  Epochs since last improvement: 3\n",
            "2022-04-08 15:21:42 [INFO ]  Evaluation at Epoch 9\n",
            "2022-04-08 15:21:42 [INFO ]  Validation accuracy: 98.51\tmacro_fscore: 98.50\tmicro_fscore: 98.51\tTPR_GAP: 1.01\tFPR_GAP: 0.14\tPPR_GAP: 0.57\t\n",
            "2022-04-08 15:21:42 [INFO ]  Test accuracy: 98.99\tmacro_fscore: 98.98\tmicro_fscore: 98.99\tTPR_GAP: 1.03\tFPR_GAP: 0.14\tPPR_GAP: 0.57\t\n",
            "2022-04-08 15:21:43 [INFO ]  Epoch:   10 [      0/  50000 ( 0%)]\tLoss: 0.0045\t Data Time: 0.01s\tTrain Time: 0.63s\n",
            "2022-04-08 15:22:19 [INFO ]  Evaluation at Epoch 10\n",
            "2022-04-08 15:22:19 [INFO ]  Validation accuracy: 98.75\tmacro_fscore: 98.74\tmicro_fscore: 98.75\tTPR_GAP: 1.02\tFPR_GAP: 0.11\tPPR_GAP: 0.59\t\n",
            "2022-04-08 15:22:19 [INFO ]  Test accuracy: 99.02\tmacro_fscore: 99.01\tmicro_fscore: 99.02\tTPR_GAP: 1.08\tFPR_GAP: 0.12\tPPR_GAP: 0.54\t\n",
            "2022-04-08 15:22:20 [INFO ]  Epoch:   11 [      0/  50000 ( 0%)]\tLoss: 0.0023\t Data Time: 0.01s\tTrain Time: 0.64s\n",
            "2022-04-08 15:22:53 [INFO ]  Epochs since last improvement: 1\n",
            "2022-04-08 15:22:56 [INFO ]  Evaluation at Epoch 11\n",
            "2022-04-08 15:22:56 [INFO ]  Validation accuracy: 98.66\tmacro_fscore: 98.65\tmicro_fscore: 98.66\tTPR_GAP: 1.38\tFPR_GAP: 0.14\tPPR_GAP: 0.58\t\n",
            "2022-04-08 15:22:56 [INFO ]  Test accuracy: 99.03\tmacro_fscore: 99.03\tmicro_fscore: 99.03\tTPR_GAP: 0.91\tFPR_GAP: 0.11\tPPR_GAP: 0.50\t\n",
            "2022-04-08 15:22:56 [INFO ]  Epoch:   12 [      0/  50000 ( 0%)]\tLoss: 0.0019\t Data Time: 0.01s\tTrain Time: 0.63s\n",
            "2022-04-08 15:23:30 [INFO ]  Epochs since last improvement: 2\n",
            "2022-04-08 15:23:32 [INFO ]  Evaluation at Epoch 12\n",
            "2022-04-08 15:23:32 [INFO ]  Validation accuracy: 98.71\tmacro_fscore: 98.70\tmicro_fscore: 98.71\tTPR_GAP: 1.19\tFPR_GAP: 0.10\tPPR_GAP: 0.57\t\n",
            "2022-04-08 15:23:32 [INFO ]  Test accuracy: 99.08\tmacro_fscore: 99.07\tmicro_fscore: 99.08\tTPR_GAP: 0.94\tFPR_GAP: 0.10\tPPR_GAP: 0.53\t\n",
            "2022-04-08 15:23:33 [INFO ]  Epoch:   13 [      0/  50000 ( 0%)]\tLoss: 0.0011\t Data Time: 0.01s\tTrain Time: 0.64s\n",
            "2022-04-08 15:24:07 [INFO ]  Epochs since last improvement: 3\n",
            "2022-04-08 15:24:09 [INFO ]  Evaluation at Epoch 13\n",
            "2022-04-08 15:24:09 [INFO ]  Validation accuracy: 98.79\tmacro_fscore: 98.78\tmicro_fscore: 98.79\tTPR_GAP: 1.05\tFPR_GAP: 0.12\tPPR_GAP: 0.58\t\n",
            "2022-04-08 15:24:09 [INFO ]  Test accuracy: 99.16\tmacro_fscore: 99.15\tmicro_fscore: 99.16\tTPR_GAP: 0.98\tFPR_GAP: 0.10\tPPR_GAP: 0.55\t\n",
            "2022-04-08 15:24:10 [INFO ]  Epoch:   14 [      0/  50000 ( 0%)]\tLoss: 0.0004\t Data Time: 0.01s\tTrain Time: 0.64s\n",
            "2022-04-08 15:24:43 [INFO ]  Epochs since last improvement: 4\n",
            "2022-04-08 15:24:46 [INFO ]  Evaluation at Epoch 14\n",
            "2022-04-08 15:24:46 [INFO ]  Validation accuracy: 98.78\tmacro_fscore: 98.77\tmicro_fscore: 98.78\tTPR_GAP: 1.25\tFPR_GAP: 0.12\tPPR_GAP: 0.57\t\n",
            "2022-04-08 15:24:46 [INFO ]  Test accuracy: 99.22\tmacro_fscore: 99.22\tmicro_fscore: 99.22\tTPR_GAP: 0.83\tFPR_GAP: 0.10\tPPR_GAP: 0.51\t\n",
            "2022-04-08 15:24:46 [INFO ]  Epoch:   15 [      0/  50000 ( 0%)]\tLoss: 0.0002\t Data Time: 0.01s\tTrain Time: 0.64s\n",
            "2022-04-08 15:25:20 [INFO ]  Epochs since last improvement: 5\n",
            "2022-04-08 15:25:22 [INFO ]  Evaluation at Epoch 15\n",
            "2022-04-08 15:25:22 [INFO ]  Validation accuracy: 98.78\tmacro_fscore: 98.77\tmicro_fscore: 98.78\tTPR_GAP: 1.17\tFPR_GAP: 0.12\tPPR_GAP: 0.59\t\n",
            "2022-04-08 15:25:22 [INFO ]  Test accuracy: 99.20\tmacro_fscore: 99.20\tmicro_fscore: 99.20\tTPR_GAP: 0.87\tFPR_GAP: 0.11\tPPR_GAP: 0.54\t\n"
          ]
        }
      ],
      "source": [
        "model.train_self(\n",
        "    train_generator = customized_training_generator,\n",
        "    dev_generator = customized_validation_generator,\n",
        "    test_generator = customized_test_generator,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xmqdlUG6UDT4",
        "outputId": "d7c6bc78-c5a4-4ba0-a48a-b6cd32be0e97"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2022-04-08 15:25:37 [INFO ]  Unexpected args: ['-f', '/root/.local/share/jupyter/runtime/kernel-9c3335e9-652e-4f42-97e0-8b86721e4921.json']\n",
            "2022-04-08 15:25:37 [INFO ]  Logging to ./results/dev/MNIST/BT_Adv/output.log\n",
            "2022-04-08 15:25:37 [INFO ]  ======================================== 2022-04-08 15:25:37 ========================================\n",
            "2022-04-08 15:25:37 [INFO ]  Base directory is ./results/dev/MNIST/BT_Adv\n",
            "Not implemented\n",
            "2022-04-08 15:25:37 [INFO ]  dataloaders need to be initialized!\n",
            "2022-04-08 15:25:37 [INFO ]  SubDiscriminator( \n",
            "2022-04-08 15:25:37 [INFO ]    (grad_rev): GradientReversal()\n",
            "2022-04-08 15:25:37 [INFO ]    (output_layer): Linear(in_features=300, out_features=2, bias=True)\n",
            "2022-04-08 15:25:37 [INFO ]    (AF): ReLU()\n",
            "2022-04-08 15:25:37 [INFO ]    (hidden_layers): ModuleList(\n",
            "2022-04-08 15:25:37 [INFO ]      (0): Linear(in_features=300, out_features=300, bias=True)\n",
            "2022-04-08 15:25:37 [INFO ]      (1): ReLU()\n",
            "2022-04-08 15:25:37 [INFO ]      (2): Linear(in_features=300, out_features=300, bias=True)\n",
            "2022-04-08 15:25:37 [INFO ]      (3): ReLU()\n",
            "2022-04-08 15:25:37 [INFO ]    )\n",
            "2022-04-08 15:25:37 [INFO ]    (criterion): CrossEntropyLoss()\n",
            "2022-04-08 15:25:37 [INFO ]  )\n",
            "2022-04-08 15:25:37 [INFO ]  Total number of parameters: 181202 \n",
            "\n",
            "2022-04-08 15:25:38 [INFO ]  Discriminator built!\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/fairlib/src/dataloaders/utils.py:163: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
            "  self.X = np.array(self.X)\n",
            "/usr/local/lib/python3.7/dist-packages/fairlib/src/dataloaders/utils.py:211: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
            "  self.X = np.array(X)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded data shapes: (20018,), (20018,), (20018,)\n",
            "Loaded data shapes: (9794,), (9794,), (9794,)\n",
            "Loaded data shapes: (9798,), (9798,), (9798,)\n",
            "2022-04-08 15:25:51 [INFO ]  MLP( \n",
            "2022-04-08 15:25:51 [INFO ]    (output_layer): Linear(in_features=300, out_features=10, bias=True)\n",
            "2022-04-08 15:25:51 [INFO ]    (AF): Tanh()\n",
            "2022-04-08 15:25:51 [INFO ]    (hidden_layers): ModuleList(\n",
            "2022-04-08 15:25:51 [INFO ]      (0): Linear(in_features=800, out_features=300, bias=True)\n",
            "2022-04-08 15:25:51 [INFO ]      (1): Tanh()\n",
            "2022-04-08 15:25:51 [INFO ]      (2): Linear(in_features=300, out_features=300, bias=True)\n",
            "2022-04-08 15:25:51 [INFO ]      (3): Tanh()\n",
            "2022-04-08 15:25:51 [INFO ]    )\n",
            "2022-04-08 15:25:51 [INFO ]    (criterion): CrossEntropyLoss()\n",
            "2022-04-08 15:25:51 [INFO ]  )\n",
            "2022-04-08 15:25:51 [INFO ]  Total number of parameters: 333610 \n",
            "\n",
            "2022-04-08 15:25:51 [INFO ]  ConvNet( \n",
            "2022-04-08 15:25:51 [INFO ]    (conv1): Conv2d(3, 20, kernel_size=(5, 5), stride=(1, 1))\n",
            "2022-04-08 15:25:51 [INFO ]    (conv2): Conv2d(20, 50, kernel_size=(5, 5), stride=(1, 1))\n",
            "2022-04-08 15:25:51 [INFO ]    (classifier): MLP(\n",
            "2022-04-08 15:25:51 [INFO ]      (output_layer): Linear(in_features=300, out_features=10, bias=True)\n",
            "2022-04-08 15:25:51 [INFO ]      (AF): Tanh()\n",
            "2022-04-08 15:25:51 [INFO ]      (hidden_layers): ModuleList(\n",
            "2022-04-08 15:25:51 [INFO ]        (0): Linear(in_features=800, out_features=300, bias=True)\n",
            "2022-04-08 15:25:51 [INFO ]        (1): Tanh()\n",
            "2022-04-08 15:25:51 [INFO ]        (2): Linear(in_features=300, out_features=300, bias=True)\n",
            "2022-04-08 15:25:51 [INFO ]        (3): Tanh()\n",
            "2022-04-08 15:25:51 [INFO ]      )\n",
            "2022-04-08 15:25:51 [INFO ]      (criterion): CrossEntropyLoss()\n",
            "2022-04-08 15:25:51 [INFO ]    )\n",
            "2022-04-08 15:25:51 [INFO ]    (criterion): CrossEntropyLoss()\n",
            "2022-04-08 15:25:51 [INFO ]  )\n",
            "2022-04-08 15:25:51 [INFO ]  Total number of parameters: 360180 \n",
            "\n"
          ]
        }
      ],
      "source": [
        "debiasing_args = {\n",
        "    \"dataset\":Shared_options[\"dataset\"], \n",
        "    \"data_dir\":Shared_options[\"data_dir\"],\n",
        "    \"device_id\":Shared_options[\"device_id\"],\n",
        "\n",
        "    # Give a name to the exp, which will be used in the path\n",
        "    \"exp_id\":\"BT_Adv\",\n",
        "\n",
        "    \"emb_size\": 4*4*50,\n",
        "    \"num_classes\": 10,\n",
        "    \"num_groups\": 2,\n",
        "\n",
        "    # Perform adversarial training if True\n",
        "    \"adv_debiasing\":True,\n",
        "\n",
        "    # Specify the hyperparameters for Balanced Training\n",
        "    \"BT\":\"Resampling\",\n",
        "    \"BTObj\":\"EO\",\n",
        "}\n",
        "\n",
        "# Init the argument\n",
        "debias_options = BaseOptions()\n",
        "debias_state = debias_options.get_state(args=debiasing_args, silence=True)\n",
        "\n",
        "customized_train_data = CustomizedDataset(args=debias_state, split=\"train\")\n",
        "customized_dev_data = CustomizedDataset(args=debias_state, split=\"dev\")\n",
        "customized_test_data = CustomizedDataset(args=debias_state, split=\"test\")\n",
        "\n",
        "# DataLoader Parameters\n",
        "tran_dataloader_params = {\n",
        "        'batch_size': state.batch_size,\n",
        "        'shuffle': True,\n",
        "        'num_workers': state.num_workers}\n",
        "\n",
        "eval_dataloader_params = {\n",
        "        'batch_size': state.test_batch_size,\n",
        "        'shuffle': False,\n",
        "        'num_workers': state.num_workers}\n",
        "\n",
        "# init dataloader\n",
        "customized_training_generator = torch.utils.data.DataLoader(customized_train_data, **tran_dataloader_params)\n",
        "customized_validation_generator = torch.utils.data.DataLoader(customized_dev_data, **eval_dataloader_params)\n",
        "customized_test_generator = torch.utils.data.DataLoader(customized_test_data, **eval_dataloader_params)\n",
        "\n",
        "debias_model = ConvNet(debias_state)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jLJQQzilUDT4",
        "outputId": "6dcb4319-222e-41b2-dad9-eb8e23cefcdd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2022-04-08 15:26:01 [INFO ]  Epoch:    0 [      0/  20018 ( 0%)]\tLoss: 1.6324\t Data Time: 0.02s\tTrain Time: 2.12s\n",
            "2022-04-08 15:26:40 [INFO ]  Evaluation at Epoch 0\n",
            "2022-04-08 15:26:40 [INFO ]  Validation accuracy: 91.53\tmacro_fscore: 91.46\tmicro_fscore: 91.53\tTPR_GAP: 5.13\tFPR_GAP: 0.75\tPPR_GAP: 1.08\t\n",
            "2022-04-08 15:26:40 [INFO ]  Test accuracy: 92.26\tmacro_fscore: 92.18\tmicro_fscore: 92.26\tTPR_GAP: 4.50\tFPR_GAP: 0.50\tPPR_GAP: 0.83\t\n",
            "2022-04-08 15:26:42 [INFO ]  Epoch:    1 [      0/  20018 ( 0%)]\tLoss: -0.4085\t Data Time: 0.01s\tTrain Time: 1.82s\n",
            "2022-04-08 15:27:21 [INFO ]  Evaluation at Epoch 1\n",
            "2022-04-08 15:27:21 [INFO ]  Validation accuracy: 94.82\tmacro_fscore: 94.77\tmicro_fscore: 94.82\tTPR_GAP: 4.61\tFPR_GAP: 0.52\tPPR_GAP: 0.69\t\n",
            "2022-04-08 15:27:21 [INFO ]  Test accuracy: 95.24\tmacro_fscore: 95.18\tmicro_fscore: 95.24\tTPR_GAP: 3.36\tFPR_GAP: 0.49\tPPR_GAP: 0.63\t\n",
            "2022-04-08 15:27:23 [INFO ]  Epoch:    2 [      0/  20018 ( 0%)]\tLoss: -0.5775\t Data Time: 0.01s\tTrain Time: 1.79s\n",
            "2022-04-08 15:28:02 [INFO ]  Evaluation at Epoch 2\n",
            "2022-04-08 15:28:02 [INFO ]  Validation accuracy: 96.91\tmacro_fscore: 96.89\tmicro_fscore: 96.91\tTPR_GAP: 1.45\tFPR_GAP: 0.11\tPPR_GAP: 0.21\t\n",
            "2022-04-08 15:28:02 [INFO ]  Test accuracy: 96.94\tmacro_fscore: 96.92\tmicro_fscore: 96.94\tTPR_GAP: 1.21\tFPR_GAP: 0.11\tPPR_GAP: 0.20\t\n",
            "2022-04-08 15:28:03 [INFO ]  Epoch:    3 [      0/  20018 ( 0%)]\tLoss: -0.5939\t Data Time: 0.01s\tTrain Time: 1.74s\n",
            "2022-04-08 15:28:42 [INFO ]  Evaluation at Epoch 3\n",
            "2022-04-08 15:28:42 [INFO ]  Validation accuracy: 97.77\tmacro_fscore: 97.76\tmicro_fscore: 97.77\tTPR_GAP: 1.13\tFPR_GAP: 0.13\tPPR_GAP: 0.20\t\n",
            "2022-04-08 15:28:42 [INFO ]  Test accuracy: 98.15\tmacro_fscore: 98.15\tmicro_fscore: 98.15\tTPR_GAP: 1.04\tFPR_GAP: 0.13\tPPR_GAP: 0.19\t\n",
            "2022-04-08 15:28:43 [INFO ]  Epoch:    4 [      0/  20018 ( 0%)]\tLoss: -0.6504\t Data Time: 0.01s\tTrain Time: 1.73s\n",
            "2022-04-08 15:29:22 [INFO ]  Evaluation at Epoch 4\n",
            "2022-04-08 15:29:22 [INFO ]  Validation accuracy: 97.92\tmacro_fscore: 97.91\tmicro_fscore: 97.92\tTPR_GAP: 0.85\tFPR_GAP: 0.10\tPPR_GAP: 0.14\t\n",
            "2022-04-08 15:29:22 [INFO ]  Test accuracy: 98.40\tmacro_fscore: 98.39\tmicro_fscore: 98.40\tTPR_GAP: 1.10\tFPR_GAP: 0.13\tPPR_GAP: 0.19\t\n",
            "2022-04-08 15:29:24 [INFO ]  Epoch:    5 [      0/  20018 ( 0%)]\tLoss: -0.6614\t Data Time: 0.01s\tTrain Time: 1.76s\n",
            "2022-04-08 15:30:03 [INFO ]  Evaluation at Epoch 5\n",
            "2022-04-08 15:30:03 [INFO ]  Validation accuracy: 98.09\tmacro_fscore: 98.08\tmicro_fscore: 98.09\tTPR_GAP: 0.79\tFPR_GAP: 0.13\tPPR_GAP: 0.18\t\n",
            "2022-04-08 15:30:03 [INFO ]  Test accuracy: 98.34\tmacro_fscore: 98.33\tmicro_fscore: 98.34\tTPR_GAP: 1.15\tFPR_GAP: 0.07\tPPR_GAP: 0.14\t\n",
            "2022-04-08 15:30:04 [INFO ]  Epoch:    6 [      0/  20018 ( 0%)]\tLoss: -0.6657\t Data Time: 0.01s\tTrain Time: 1.73s\n",
            "2022-04-08 15:30:45 [INFO ]  Evaluation at Epoch 6\n",
            "2022-04-08 15:30:45 [INFO ]  Validation accuracy: 98.25\tmacro_fscore: 98.24\tmicro_fscore: 98.25\tTPR_GAP: 0.72\tFPR_GAP: 0.09\tPPR_GAP: 0.11\t\n",
            "2022-04-08 15:30:45 [INFO ]  Test accuracy: 98.63\tmacro_fscore: 98.62\tmicro_fscore: 98.63\tTPR_GAP: 0.83\tFPR_GAP: 0.12\tPPR_GAP: 0.13\t\n",
            "2022-04-08 15:30:47 [INFO ]  Epoch:    7 [      0/  20018 ( 0%)]\tLoss: -0.6750\t Data Time: 0.01s\tTrain Time: 1.77s\n",
            "2022-04-08 15:31:24 [INFO ]  Epochs since last improvement: 1\n",
            "2022-04-08 15:31:27 [INFO ]  Evaluation at Epoch 7\n",
            "2022-04-08 15:31:27 [INFO ]  Validation accuracy: 98.19\tmacro_fscore: 98.18\tmicro_fscore: 98.19\tTPR_GAP: 0.96\tFPR_GAP: 0.09\tPPR_GAP: 0.14\t\n",
            "2022-04-08 15:31:27 [INFO ]  Test accuracy: 98.59\tmacro_fscore: 98.58\tmicro_fscore: 98.59\tTPR_GAP: 0.68\tFPR_GAP: 0.11\tPPR_GAP: 0.14\t\n",
            "2022-04-08 15:31:28 [INFO ]  Epoch:    8 [      0/  20018 ( 0%)]\tLoss: -0.6807\t Data Time: 0.01s\tTrain Time: 1.77s\n",
            "2022-04-08 15:32:04 [INFO ]  Epochs since last improvement: 2\n",
            "2022-04-08 15:32:07 [INFO ]  Evaluation at Epoch 8\n",
            "2022-04-08 15:32:07 [INFO ]  Validation accuracy: 98.28\tmacro_fscore: 98.27\tmicro_fscore: 98.28\tTPR_GAP: 0.77\tFPR_GAP: 0.07\tPPR_GAP: 0.11\t\n",
            "2022-04-08 15:32:07 [INFO ]  Test accuracy: 98.40\tmacro_fscore: 98.39\tmicro_fscore: 98.40\tTPR_GAP: 0.82\tFPR_GAP: 0.11\tPPR_GAP: 0.13\t\n",
            "2022-04-08 15:32:09 [INFO ]  Epoch:    9 [      0/  20018 ( 0%)]\tLoss: -0.6773\t Data Time: 0.01s\tTrain Time: 1.75s\n",
            "2022-04-08 15:32:47 [INFO ]  Evaluation at Epoch 9\n",
            "2022-04-08 15:32:47 [INFO ]  Validation accuracy: 98.45\tmacro_fscore: 98.44\tmicro_fscore: 98.45\tTPR_GAP: 0.81\tFPR_GAP: 0.08\tPPR_GAP: 0.11\t\n",
            "2022-04-08 15:32:47 [INFO ]  Test accuracy: 98.80\tmacro_fscore: 98.79\tmicro_fscore: 98.80\tTPR_GAP: 0.57\tFPR_GAP: 0.10\tPPR_GAP: 0.10\t\n",
            "2022-04-08 15:32:49 [INFO ]  Epoch:   10 [      0/  20018 ( 0%)]\tLoss: -0.6867\t Data Time: 0.01s\tTrain Time: 1.77s\n",
            "2022-04-08 15:33:25 [INFO ]  Epochs since last improvement: 1\n",
            "2022-04-08 15:33:28 [INFO ]  Evaluation at Epoch 10\n",
            "2022-04-08 15:33:28 [INFO ]  Validation accuracy: 98.50\tmacro_fscore: 98.48\tmicro_fscore: 98.50\tTPR_GAP: 0.61\tFPR_GAP: 0.06\tPPR_GAP: 0.09\t\n",
            "2022-04-08 15:33:28 [INFO ]  Test accuracy: 98.77\tmacro_fscore: 98.76\tmicro_fscore: 98.77\tTPR_GAP: 0.65\tFPR_GAP: 0.08\tPPR_GAP: 0.09\t\n",
            "2022-04-08 15:33:29 [INFO ]  Epoch:   11 [      0/  20018 ( 0%)]\tLoss: -0.6892\t Data Time: 0.01s\tTrain Time: 1.76s\n",
            "2022-04-08 15:34:05 [INFO ]  Epochs since last improvement: 2\n",
            "2022-04-08 15:34:08 [INFO ]  Evaluation at Epoch 11\n",
            "2022-04-08 15:34:08 [INFO ]  Validation accuracy: 98.34\tmacro_fscore: 98.32\tmicro_fscore: 98.34\tTPR_GAP: 0.79\tFPR_GAP: 0.09\tPPR_GAP: 0.11\t\n",
            "2022-04-08 15:34:08 [INFO ]  Test accuracy: 98.67\tmacro_fscore: 98.66\tmicro_fscore: 98.67\tTPR_GAP: 0.80\tFPR_GAP: 0.09\tPPR_GAP: 0.13\t\n",
            "2022-04-08 15:34:10 [INFO ]  Epoch:   12 [      0/  20018 ( 0%)]\tLoss: -0.6875\t Data Time: 0.01s\tTrain Time: 1.77s\n",
            "2022-04-08 15:34:49 [INFO ]  Evaluation at Epoch 12\n",
            "2022-04-08 15:34:49 [INFO ]  Validation accuracy: 98.70\tmacro_fscore: 98.69\tmicro_fscore: 98.70\tTPR_GAP: 0.64\tFPR_GAP: 0.07\tPPR_GAP: 0.11\t\n",
            "2022-04-08 15:34:49 [INFO ]  Test accuracy: 98.77\tmacro_fscore: 98.76\tmicro_fscore: 98.77\tTPR_GAP: 0.80\tFPR_GAP: 0.09\tPPR_GAP: 0.14\t\n",
            "2022-04-08 15:34:51 [INFO ]  Epoch:   13 [      0/  20018 ( 0%)]\tLoss: -0.6881\t Data Time: 0.01s\tTrain Time: 1.83s\n",
            "2022-04-08 15:35:26 [INFO ]  Epochs since last improvement: 1\n",
            "2022-04-08 15:35:29 [INFO ]  Evaluation at Epoch 13\n",
            "2022-04-08 15:35:29 [INFO ]  Validation accuracy: 98.40\tmacro_fscore: 98.38\tmicro_fscore: 98.40\tTPR_GAP: 0.69\tFPR_GAP: 0.07\tPPR_GAP: 0.09\t\n",
            "2022-04-08 15:35:29 [INFO ]  Test accuracy: 98.64\tmacro_fscore: 98.63\tmicro_fscore: 98.64\tTPR_GAP: 0.82\tFPR_GAP: 0.09\tPPR_GAP: 0.13\t\n",
            "2022-04-08 15:35:31 [INFO ]  Epoch:   14 [      0/  20018 ( 0%)]\tLoss: -0.6886\t Data Time: 0.01s\tTrain Time: 1.75s\n",
            "2022-04-08 15:36:06 [INFO ]  Epochs since last improvement: 2\n",
            "2022-04-08 15:36:09 [INFO ]  Evaluation at Epoch 14\n",
            "2022-04-08 15:36:09 [INFO ]  Validation accuracy: 98.49\tmacro_fscore: 98.48\tmicro_fscore: 98.49\tTPR_GAP: 0.75\tFPR_GAP: 0.10\tPPR_GAP: 0.14\t\n",
            "2022-04-08 15:36:09 [INFO ]  Test accuracy: 98.77\tmacro_fscore: 98.75\tmicro_fscore: 98.77\tTPR_GAP: 0.79\tFPR_GAP: 0.07\tPPR_GAP: 0.10\t\n",
            "2022-04-08 15:36:11 [INFO ]  Epoch:   15 [      0/  20018 ( 0%)]\tLoss: -0.6904\t Data Time: 0.01s\tTrain Time: 1.74s\n",
            "2022-04-08 15:36:47 [INFO ]  Epochs since last improvement: 3\n",
            "2022-04-08 15:36:50 [INFO ]  Evaluation at Epoch 15\n",
            "2022-04-08 15:36:50 [INFO ]  Validation accuracy: 98.48\tmacro_fscore: 98.46\tmicro_fscore: 98.48\tTPR_GAP: 0.59\tFPR_GAP: 0.08\tPPR_GAP: 0.12\t\n",
            "2022-04-08 15:36:50 [INFO ]  Test accuracy: 98.80\tmacro_fscore: 98.79\tmicro_fscore: 98.80\tTPR_GAP: 0.68\tFPR_GAP: 0.07\tPPR_GAP: 0.10\t\n",
            "2022-04-08 15:36:52 [INFO ]  Epoch:   16 [      0/  20018 ( 0%)]\tLoss: -0.6925\t Data Time: 0.01s\tTrain Time: 1.75s\n",
            "2022-04-08 15:37:27 [INFO ]  Epochs since last improvement: 4\n",
            "2022-04-08 15:37:30 [INFO ]  Evaluation at Epoch 16\n",
            "2022-04-08 15:37:30 [INFO ]  Validation accuracy: 98.48\tmacro_fscore: 98.47\tmicro_fscore: 98.48\tTPR_GAP: 0.61\tFPR_GAP: 0.06\tPPR_GAP: 0.10\t\n",
            "2022-04-08 15:37:30 [INFO ]  Test accuracy: 98.73\tmacro_fscore: 98.73\tmicro_fscore: 98.73\tTPR_GAP: 0.73\tFPR_GAP: 0.07\tPPR_GAP: 0.10\t\n",
            "2022-04-08 15:37:32 [INFO ]  Epoch:   17 [      0/  20018 ( 0%)]\tLoss: -0.6918\t Data Time: 0.02s\tTrain Time: 1.74s\n",
            "2022-04-08 15:38:08 [INFO ]  Epochs since last improvement: 5\n",
            "2022-04-08 15:38:11 [INFO ]  Evaluation at Epoch 17\n",
            "2022-04-08 15:38:11 [INFO ]  Validation accuracy: 98.62\tmacro_fscore: 98.61\tmicro_fscore: 98.62\tTPR_GAP: 0.65\tFPR_GAP: 0.07\tPPR_GAP: 0.09\t\n",
            "2022-04-08 15:38:11 [INFO ]  Test accuracy: 98.85\tmacro_fscore: 98.84\tmicro_fscore: 98.85\tTPR_GAP: 0.70\tFPR_GAP: 0.08\tPPR_GAP: 0.10\t\n"
          ]
        }
      ],
      "source": [
        "debias_model.train_self(\n",
        "    train_generator = customized_training_generator,\n",
        "    dev_generator = customized_validation_generator,\n",
        "    test_generator = customized_test_generator,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ml19ZVl9UDT4"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "colored_MNIST.ipynb",
      "provenance": []
    },
    "interpreter": {
      "hash": "086b6e2ac5931a4f7a51b812f0475290a145e7d51f3e22b53c52adf5c273fe30"
    },
    "kernelspec": {
      "display_name": "Python 3.7.12 ('py37')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.12"
    },
    "orig_nbformat": 4,
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "013929ca9c3f4d6485f9546f42b5ebf7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "03591eb69c4c4e4cb206b50ee7a20f04": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c359274f675a47f391c95666dc2b895b",
              "IPY_MODEL_2e00fdb819f64bada76a0aaeb227ffee",
              "IPY_MODEL_83dc7f3f068f4e94b4a0ee4ae847adc4"
            ],
            "layout": "IPY_MODEL_f261238955694ef28f1a126bde95cb3a"
          }
        },
        "072b3db410264da2a45c2aea43800d80": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1603eba4355a46e9880368c7d3897ef7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_883283fb098c4dd5a04aa7b8a2b09c72",
            "max": 28881,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fbbae0dfa4eb461ca0d4271f9f0445fb",
            "value": 28881
          }
        },
        "16f0e0a83d6248839141efc74caf7369": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_830fd908972f4b2cb8a90e9630b557a0",
              "IPY_MODEL_fd2bdd89b7914b11b6e3263342f4283f",
              "IPY_MODEL_62580203dd0b4bb0adb89e97cd0aaabc"
            ],
            "layout": "IPY_MODEL_b8a4e26a95c74fb5aa4f0ca1309f8e35"
          }
        },
        "18a2bcdf5333492b8a5613ac67b1d8c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1e74b9dc176e4699a4e2289fbfdddca8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1fccbc6dc6634631b3ca89a553e76ef4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "215b63d0bb1f43f99449b822a37409a7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2df81f5b4cc6454f9936fd0c85420054": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2e00fdb819f64bada76a0aaeb227ffee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3c1b9bc1eac34136ac4f469bed1657b9",
            "max": 9912422,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2df81f5b4cc6454f9936fd0c85420054",
            "value": 9912422
          }
        },
        "36771f09904f4667bef18535967c88fc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "39f5167c8c3f403dbeed7fe731d7b16f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3c1b9bc1eac34136ac4f469bed1657b9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3ca33feeb1754b98919c4496f2da75bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_65dd39f405c24b429e399f6574698eb6",
              "IPY_MODEL_1603eba4355a46e9880368c7d3897ef7",
              "IPY_MODEL_62e7f79f3a694ee090b8aa189d235b6a"
            ],
            "layout": "IPY_MODEL_816b7a24e6334f3aa5670fbac13019da"
          }
        },
        "4ac07f673f2b470c89eabd90a979402b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4cf56ce8a4eb4042a738868bcc7536dd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5489240ca06247b98e3b476d2aa994ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5979d3c990ae425ebfb42c39cf9ff0b5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "62580203dd0b4bb0adb89e97cd0aaabc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ea8687c6368f423e860e213215e6209c",
            "placeholder": "​",
            "style": "IPY_MODEL_18a2bcdf5333492b8a5613ac67b1d8c3",
            "value": " 1649664/? [00:00&lt;00:00, 4252487.46it/s]"
          }
        },
        "62e7f79f3a694ee090b8aa189d235b6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_215b63d0bb1f43f99449b822a37409a7",
            "placeholder": "​",
            "style": "IPY_MODEL_072b3db410264da2a45c2aea43800d80",
            "value": " 29696/? [00:00&lt;00:00, 516354.71it/s]"
          }
        },
        "65dd39f405c24b429e399f6574698eb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4cf56ce8a4eb4042a738868bcc7536dd",
            "placeholder": "​",
            "style": "IPY_MODEL_d6d44a123d4b4ee69dcbe22623048517",
            "value": ""
          }
        },
        "816b7a24e6334f3aa5670fbac13019da": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "830fd908972f4b2cb8a90e9630b557a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_36771f09904f4667bef18535967c88fc",
            "placeholder": "​",
            "style": "IPY_MODEL_961e1bfb58e04db79fb2444b79584db2",
            "value": ""
          }
        },
        "83dc7f3f068f4e94b4a0ee4ae847adc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_013929ca9c3f4d6485f9546f42b5ebf7",
            "placeholder": "​",
            "style": "IPY_MODEL_976c70ed80de4f75a88e5b4e96b55337",
            "value": " 9913344/? [00:01&lt;00:00, 6111909.20it/s]"
          }
        },
        "876051a51652431b8dbe3ded0451a326": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "883283fb098c4dd5a04aa7b8a2b09c72": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93571cc249894cb984d8e96b8b6961e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4ac07f673f2b470c89eabd90a979402b",
            "max": 4542,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1fccbc6dc6634631b3ca89a553e76ef4",
            "value": 4542
          }
        },
        "961e1bfb58e04db79fb2444b79584db2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "976c70ed80de4f75a88e5b4e96b55337": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a29fcb4a206543e4bd203a664ae8c501": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aef4e5657f454247b7edc5e930827246",
            "placeholder": "​",
            "style": "IPY_MODEL_876051a51652431b8dbe3ded0451a326",
            "value": " 5120/? [00:00&lt;00:00, 107199.98it/s]"
          }
        },
        "aef4e5657f454247b7edc5e930827246": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b616a7a1e329469f8d6e5ab3d1e7a154": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_39f5167c8c3f403dbeed7fe731d7b16f",
            "placeholder": "​",
            "style": "IPY_MODEL_5489240ca06247b98e3b476d2aa994ee",
            "value": ""
          }
        },
        "b8a4e26a95c74fb5aa4f0ca1309f8e35": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c359274f675a47f391c95666dc2b895b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d367c12acbbf411abc8062017db50ebf",
            "placeholder": "​",
            "style": "IPY_MODEL_1e74b9dc176e4699a4e2289fbfdddca8",
            "value": ""
          }
        },
        "d367c12acbbf411abc8062017db50ebf": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d608093e67d84b61ada485ea0825e678": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d6d44a123d4b4ee69dcbe22623048517": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ea8687c6368f423e860e213215e6209c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "efacb89fe81f4653bb2bf550999d6b74": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f261238955694ef28f1a126bde95cb3a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "faab9ab370f74de2939729d8c2959fd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b616a7a1e329469f8d6e5ab3d1e7a154",
              "IPY_MODEL_93571cc249894cb984d8e96b8b6961e0",
              "IPY_MODEL_a29fcb4a206543e4bd203a664ae8c501"
            ],
            "layout": "IPY_MODEL_efacb89fe81f4653bb2bf550999d6b74"
          }
        },
        "fbbae0dfa4eb461ca0d4271f9f0445fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fd2bdd89b7914b11b6e3263342f4283f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5979d3c990ae425ebfb42c39cf9ff0b5",
            "max": 1648877,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d608093e67d84b61ada485ea0825e678",
            "value": 1648877
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
